#!/usr/bin/env node
var fs                  = require('fs');
var readline            = require('readline');
var http                = require('http');
var utils               = require('./utils');
var pipeline            = require('when/pipeline');
var sequence            = require('when/sequence');
var when                = require('when');
var exec                = require('child_process').exec
var path                = require('path');
var TSVParser           = require('./tsvparser').TSVParser;
var JSONify             = require('./jsonify').JSONify;
var Config              = require('./config').Config;
var entries             = 0;
var dotEvery            = 10000;

exports.Wiktionary = ( function() {

    /*
    params : {
        url         : where the source dictionary is
        output      : the name of the folder where to put the results (folder will be created if not existing)
        input       : where freq files and 7zip will be looked for
        launguages  : optional: filter in specific languages, e.g.: ['Italian']
    }
    */
    function Wiktionary(params) {
        this.params = params;
        this.params.output = {
            last : this.params.output+"/last",
            best : this.params.output+"/best"
        }
        if (!fs.existsSync(params.scratch)) {
            fs.mkdirSync(params.scratch);
        }
    }



    Wiktionary.prototype.downloaded = function() {
        var result = when.defer();
        var resultFile = this.params.scratch+"/all.tsv.gz";
        if (!fs.existsSync(resultFile)) {
            console.log("Wiktionary : downloading \t:'"+resultFile+"' (all languages dump file)");
            var chunks = 0;
            var file = fs.createWriteStream(resultFile);
            var request = http.get(this.params.url, function(response) {
                response.pipe(file);
                file.on('finish', function() {
                    file.close();
                    result.resolve(resultFile);
                });
            });
        } else {
            result.resolve(resultFile);
        }
        return result.promise;
    }

    function unzipLinux(self, zipped, resultFile) {
        var result = when.defer();
        var child = exec('gunzip '+zipped, function (error, stdout, stderr) {
            if (error !== null) {
                result.reject(error);
            } else {
                result.resolve(zipped.substring(0, zipped.lastIndexOf(".")));
            }
        });
        return result.promise;
    }


    function unzipWin(self, zipped, resultFile) {
        var unzipRaceFolder = self.params.scratch+"/temp";
        if (fs.existsSync(unzipRaceFolder)) {
            utils.rmdirSyncForce(unzipRaceFolder);
        }
        fs.mkdirSync(unzipRaceFolder);

        var trials = [
            "7za.exe -o%OUTPUTFOLDER% e %ZIPPEDFILE%",
            self.params.input+"/7za.exe -o%OUTPUTFOLDER% e %ZIPPEDFILE%",
        ];

        function tryUnzip(zipcommand, resolved) {
            var result = when.defer();
            var child = exec(zipcommand, function (error, stdout, stderr) {
                if (error !== null) {
                    result.reject(error);
                } else {
                    result.resolve(resolved);
                }
            });
            return result.promise;
        }

        function addTrial(index) {
            trials[index] = tryUnzip(
                trials[index]
                    .replace("%OUTPUTFOLDER%", unzipRaceFolder+"/"+index)
                    .replace("%ZIPPEDFILE%", zipped),
                unzipRaceFolder+"/"+index
            );
        }

        function extract(srcFolder, dstFile) {
            var files = fs.readdirSync(srcFolder);
            if (!files || !files.length == 1) {
                throw new Error("Expected exactly one file after unzipping wiktionary file :"+zipped);
            }
            fs.renameSync(srcFolder+"/"+files[0], dstFile);
            return dstFile;
        }

        for (var i = 0; i < trials.length; i++) {
            addTrial(i);
        }

        return pipeline([
            function()          {return when.any(trials);},
            function(folder)    {return extract(folder, resultFile);}
        ]).ensure(function() {
            utils.rmdirSyncForce(unzipRaceFolder);
        });
    }


    Wiktionary.prototype.unzipped = function() {
        var self = this;
        var resultFile = this.params.scratch+"/all.tsv"
        if (fs.existsSync(resultFile)) {
            return resultFile;
        }
        return pipeline([
            function() {return self.downloaded();},
            function(zipped) {
                console.log("Wiktionary : Unzipping \t\t:'"+zipped+"'");
                if (process.platform === "linux") {
                    return unzipLinux(self, zipped, resultFile);
                } else {
                    return unzipWin(self, zipped, resultFile);
                }
            }
        ]);
    }


    Wiktionary.prototype.split = function() {
        var self = this;
        var outputFolder = self.params.scratch+"/languages/";
        if (fs.existsSync(outputFolder)) {
            var files = fs.readdirSync(outputFolder);
            if (files.length > 0) {
                return when.resolve(outputFolder);
            }
        } else {
            fs.mkdirSync(outputFolder);
        }
        return pipeline([
            function() {return self.unzipped();},
            function(unzipped) {
                console.log("Wiktionary : splitting \t\t:'"+unzipped+"'");
                var result = when.defer();
                var rd = readline.createInterface({
                    input: fs.createReadStream(unzipped),
                    output: process.stdout,
                    terminal: false
                });

                rd.on('line', function(line) {
                    var result = line.match(/^([^\s]+)\s(.+)/);
                    var language = result[1];
                    var restOfLine = result[2];
                    var languageFolder = outputFolder+language;
                    if (entries++%dotEvery == 0) {
                        process.stdout.write(".");
                    }
                    if (!fs.existsSync(languageFolder)) {
                        fs.mkdirSync(languageFolder);
                    }
                    fs.appendFileSync(languageFolder+"/raw.tsv", restOfLine + "\n");
                });

                rd.on('close', function() {
                    console.log("\nWiktionary : finished splitting '"+unzipped+"'");
                    result.resolve(outputFolder);
                });
                return result.promise;
            }
        ]);
    }




    Wiktionary.prototype.formalise = function() {
        var self = this;

        return pipeline([
            function() {
                var languages = fs.readdirSync(self.params.input+"/languages");
                for (var i = 0; i < languages.length; i++) {
                    var potentialLanuageFolder = self.params.input+"/languages/"+languages[i]+"/sample.txt";
                    if (!fs.existsSync(potentialLanuageFolder)) {
                        languages.splice(i--,1);
                    } else {
                        languages[i] = languages[i];
                    }
                }

                if (languages.length) {

                    if (!fs.existsSync(self.params.output.last+"/languages/")) {
                        utils.mkdirRecursive(self.params.output.last+"/languages/");
                    }

                    var formaliseLanguageTasks = [];
                    function addFormaliseLanguageTask(language) {
                        formaliseLanguageTasks.push(function() {
                            return formaliseLanguage(self, language);
                        });
                    }
                    for (var i = 0; i < languages.length; i++) {
                        addFormaliseLanguageTask(languages[i]);
                    }

                    return sequence(formaliseLanguageTasks);
                } else {
                    throw new Error("Wiktionary : No correctly formed language folders found in input '"+self.params.input+"/languages'");
                }
            },
            function() {
                var masterManifestFile = self.params.output.last+"/manifest.json";
                console.log("Wiktionary : All languages dumped. Writing master manifest '"+masterManifestFile+"'");
                if (fs.existsSync(masterManifestFile)) {
                    fs.unlink(masterManifestFile);
                }
                var masterManifest = {
                    timestamp : new Date().getTime(),
                    languages : {}
                };

                var languages = fs.readdirSync(self.params.output.last+"/languages");
                for (var i = 0; i < languages.length; i++) {
                    var fileName = "languages/"+languages[i]+"/";
                    masterManifest.languages[languages[i]] = fileName;
                }

                fs.appendFileSync(masterManifestFile, JSON.stringify(masterManifest, undefined, 2));
            }
        ]);
    }

    function formaliseLanguage(self, language) {
        var sampleFile = self.params.input+"/languages/"+language+"/sample.txt";
        if (!fs.existsSync(sampleFile)) {
            throw new Error("No input found :"+sampleFile);
        }
        var rawInputLocations = [
            self.params.input+"/languages/"+language+"/raw.tsv",
            self.params.scratch+"/languages/"+language+"/raw.tsv"
        ];

        var languageDictFile = rawInputLocations[0];
        return pipeline([
            function() {
                if (!fs.existsSync(languageDictFile)) {
                    languageDictFile = rawInputLocations[1];
                    return self.split();
                }
            },
            function() {
                if (!fs.existsSync(languageDictFile)) {
                    throw new Error("No input found for language '"+language+"' in :\n(1) "+rawInputLocations[0]+"\n(2) "+rawInputLocations[1]);
                }
                return doFormaliseLanguage(
                    self,
                    language,
                    {
                        languageDictFile : languageDictFile,
                        sampleFile       : sampleFile,
                        scratch          : self.params.scratch,
                        output           : self.params.output.last+"/languages/"+language
                    }
                );
            },
            function() {
                console.log("Assesing the quality of the output for language '"+language+"'");
                var lastBetter = false;
                if (!fs.existsSync(self.params.output.best+"/languages/"+language)) {
                    console.log("No previous outputs for for language '"+language+"', promoting current.");
                    lastBetter = true;
                } else {

                    var bestManifest = JSON.parse(fs.readFileSync(self.params.output.best+"/languages/"+language+"/manifest.json"));
                    var lastManifest = JSON.parse(fs.readFileSync(self.params.output.last+"/languages/"+language+"/manifest.json"));

                    var deltaRawEntries = lastManifest.stats.rawEntries-bestManifest.stats.rawEntries;
                    var deltaFinalEntries = lastManifest.stats.finalEntries-bestManifest.stats.finalEntries;
                    var deltaParsing = lastManifest.stats.errors.parsing-bestManifest.stats.errors.parsing;
                    var deltaProcessing = lastManifest.stats.errors.processing-bestManifest.stats.errors.processing;
                    var deltaMissing = lastManifest.stats.errors.missing-bestManifest.stats.errors.missing;


                    if ( deltaRawEntries > 0 ) {
                        console.log(deltaRawEntries+" more entries (than last best) has been successfully parsed");
                        var bestProcessedFraction = Math.round(10000*bestManifest.stats.errors.processing/bestManifest.stats.rawEntries)/100;
                        var lastProcessedFraction = Math.round(10000*lastManifest.stats.errors.processing/lastManifest.stats.rawEntries)/100;
                        console.log("Of which "+lastProcessedFraction+"% has been successfully processed (compared to "+bestProcessedFraction+"% for the last best)");
                        if ( lastProcessedFraction > bestProcessedFraction ) {
                            console.log("Promoting current as a new best as it has less processing errors");
                            lastBetter = true;
                        } else {
                            console.log("Leaving the current best as it has less or the same number of processing errors.");
                        }

                    } else {
                        console.log("Leaving the current best as it has more or the same number of entries (raw delta: "+deltaRawEntries+", final delta: "+deltaFinalEntries+")");
                    }
                }

                if (lastBetter) {
                    utils.copyFolder(self.params.output.last+"/languages/"+language, self.params.output.best+"/languages/"+language);
                }
            }
        ]);
    }

    function doFormaliseLanguage(self, languageName, params) {
        var jsonify = new JSONify(
            languageName,
            new TSVParser({
                input   : params.languageDictFile,
                output  : params.scratch
            }),
            new Config(self.params.input+"/languages/"+languageName),
            params
        );

        if (utils.outputStale({
            input       : [
                params.sampleFile,
                params.languageDictFile,
                module.filename]
                .concat(jsonify.dependencies),
            output      : params.output.last
        })) {
            return jsonify.process();
        } else {
            console.log("Wiktionary : language folder ("+params.output+") already up to date");
        }
    }

    return Wiktionary;
})();



